# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
import requests
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter


class LatestPoliticsSpiderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, or item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Request or item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)


class LatestPoliticsDownloaderMiddleware:

    def __init__(self, proxies):
        self.proxies = proxies

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        proxies = {"http": "http://scraperapi:@proxy-server.scraperapi.com:8001"}

        middleware = cls(proxies)
        crawler.signals.connect(middleware.spider_opened, signal=signals.spider_opened)
        return middleware

    def process_request(self, request, spider):
        request.meta['proxy'] = self.proxies.get('http')
        return None

    def process_response(self, request, response, spider):
        # 通过发出测试请求来检查代理是否正在工作
        test_url = 'http://httpbin.org/ip'
        proxy = self.proxies.get('http')

        try:
            spider.logger.info("正在使用代理进行测试请求")
            test_response = requests.get(test_url, proxies={'http': proxy}, verify=False)
            spider.logger.info("代理正在工作，返回响应")
            if test_response.status_code == 200:
                return response
            else:
                spider.logger.info("代理不起作用，如果可用，请用其他代理重试请求")
                return self._retry_request(request, spider)
        except:
            spider.logger.info(f"代理请求异常{exception}")
            return self._retry_request(request, spider)

    @staticmethod
    def _retry_request(request, spider):
        max_retry_times = 3
        retry_times = request.meta.get('retry_times', 0)

        if retry_times < max_retry_times:
            spider.logger.info("重试请求，尝试次数：%d" % (retry_times + 1))
            new_request = request.copy()
            new_request.meta['retry_times'] = retry_times + 1
            return new_request

        else:
            spider.logger.info("请求已达到最大重试次数，放弃请求")
            return None

    def process_exception(self, request, exception, spider):

        spider.logger.info("请求异常: %s" % str(exception))
        return self._retry_request(request, spider)

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)
